{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "2df0a1f692e64fdfb0cefe79815f0b72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f6f6fd67d3ba434c967461f8ccaf0d8e",
              "IPY_MODEL_1803368fc8644332b7be4b4ca60bb2a8",
              "IPY_MODEL_eebf7c12549f4abb96a514038f678bbf"
            ],
            "layout": "IPY_MODEL_5ecef016b03f4887ae6711875752b77f"
          }
        },
        "f6f6fd67d3ba434c967461f8ccaf0d8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c4895a530b424d18a5788d39dbb5e286",
            "placeholder": "​",
            "style": "IPY_MODEL_8d3775b9b66a44ff8e419186e66560f2",
            "value": "100%"
          }
        },
        "1803368fc8644332b7be4b4ca60bb2a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ebd185a979a04d639b0d98656ad6ee39",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bc0604af87f947639663a3641173a29c",
            "value": 3
          }
        },
        "eebf7c12549f4abb96a514038f678bbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7b2d74d09b104c7bb536fba61289a556",
            "placeholder": "​",
            "style": "IPY_MODEL_4d1e69159a674720ab6c059a93e5b9d3",
            "value": " 3/3 [00:00&lt;00:00, 156.10it/s]"
          }
        },
        "5ecef016b03f4887ae6711875752b77f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c4895a530b424d18a5788d39dbb5e286": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d3775b9b66a44ff8e419186e66560f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ebd185a979a04d639b0d98656ad6ee39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc0604af87f947639663a3641173a29c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7b2d74d09b104c7bb536fba61289a556": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d1e69159a674720ab6c059a93e5b9d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0079e93f38774ea59ea4e12f9026ca89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_45cedfc5ac0340ee954dbd5b8c5214e3",
              "IPY_MODEL_587c8a1ef6e4423dacfd79445eeece3e",
              "IPY_MODEL_ee1acbbf59b44c4f95281b4fcd9caa17"
            ],
            "layout": "IPY_MODEL_0ffa7eb64d5f4c57b10585e132498af8"
          }
        },
        "45cedfc5ac0340ee954dbd5b8c5214e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77000c50e754461189db32d6f2ad0504",
            "placeholder": "​",
            "style": "IPY_MODEL_d688073778854c42acf30d212b8cf051",
            "value": "Downloading data: 100%"
          }
        },
        "587c8a1ef6e4423dacfd79445eeece3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ab2ea25bb2bb4250b4797bcaa74cbd52",
            "max": 16758320,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7f8b7020948245ce96388074f0291c4b",
            "value": 16758320
          }
        },
        "ee1acbbf59b44c4f95281b4fcd9caa17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c7aa64302d1343c8bc4a039c5da3f89d",
            "placeholder": "​",
            "style": "IPY_MODEL_68a9b65ed92743798d88bb7d8c3a8337",
            "value": " 16.8M/16.8M [00:00&lt;00:00, 27.8MB/s]"
          }
        },
        "0ffa7eb64d5f4c57b10585e132498af8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77000c50e754461189db32d6f2ad0504": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d688073778854c42acf30d212b8cf051": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ab2ea25bb2bb4250b4797bcaa74cbd52": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f8b7020948245ce96388074f0291c4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c7aa64302d1343c8bc4a039c5da3f89d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68a9b65ed92743798d88bb7d8c3a8337": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "55eacaeb314c41928c656375dbec7ecd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_121c357301284a279017f21ac1d20de2",
              "IPY_MODEL_63a76ff7cb0d4f728645bfd2f6309c55",
              "IPY_MODEL_1ea4ae4b10b745dfbeea153487644c27"
            ],
            "layout": "IPY_MODEL_17aee5d00c25468c8369e091550ac030"
          }
        },
        "121c357301284a279017f21ac1d20de2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9717f61134df4bc1b03c2545d6360cb6",
            "placeholder": "​",
            "style": "IPY_MODEL_99b02bb03e564523af650c57cdb526cf",
            "value": "Generating train split: 100%"
          }
        },
        "63a76ff7cb0d4f728645bfd2f6309c55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83fd6a9bcbcb4993a253af7a4525a1bd",
            "max": 206112,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f5d58eb555b143e890a814607acee638",
            "value": 206112
          }
        },
        "1ea4ae4b10b745dfbeea153487644c27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b9b1db4ee56141cb9658fa95d56f518c",
            "placeholder": "​",
            "style": "IPY_MODEL_6593d2518a6d49038852a28abe90075c",
            "value": " 206112/206112 [00:12&lt;00:00, 97871.97 examples/s]"
          }
        },
        "17aee5d00c25468c8369e091550ac030": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "9717f61134df4bc1b03c2545d6360cb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "99b02bb03e564523af650c57cdb526cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "83fd6a9bcbcb4993a253af7a4525a1bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5d58eb555b143e890a814607acee638": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b9b1db4ee56141cb9658fa95d56f518c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6593d2518a6d49038852a28abe90075c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5ddac0c7d61a4744a77fe779251d9992": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2fbaade58f664e26a5f65514311af29c",
              "IPY_MODEL_97757d25dbee4a08ba909c067b43d675",
              "IPY_MODEL_fc299d5d05c847489b720b1f662e695b"
            ],
            "layout": "IPY_MODEL_13c3347b7bf34dda818903a5cbec0dab"
          }
        },
        "2fbaade58f664e26a5f65514311af29c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_80f668c404b34d58b1f2a9595545d1b7",
            "placeholder": "​",
            "style": "IPY_MODEL_735e20651b57481f8902e11b9050f041",
            "value": "Generating test split:   0%"
          }
        },
        "97757d25dbee4a08ba909c067b43d675": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4bf99c47ffdf447ca592a748988aeb39",
            "max": 8079,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4e36e31262e24608be8d72fcb9ee6d65",
            "value": 8079
          }
        },
        "fc299d5d05c847489b720b1f662e695b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_343d0ae065304384a33e5c07479ae724",
            "placeholder": "​",
            "style": "IPY_MODEL_2f3eca8ddec74681ac58af5a7e004160",
            "value": " 0/8079 [00:00&lt;?, ? examples/s]"
          }
        },
        "13c3347b7bf34dda818903a5cbec0dab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "80f668c404b34d58b1f2a9595545d1b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "735e20651b57481f8902e11b9050f041": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4bf99c47ffdf447ca592a748988aeb39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e36e31262e24608be8d72fcb9ee6d65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "343d0ae065304384a33e5c07479ae724": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2f3eca8ddec74681ac58af5a7e004160": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cfa511fba9a8412e83f3be96f02cbc95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2a832f046e554c00acbc7559f6104550",
              "IPY_MODEL_b4c146f0f0194dc99f8d28ea3ae01719",
              "IPY_MODEL_ce0af431d90047ea9ba4255809220a5b"
            ],
            "layout": "IPY_MODEL_c4cc841254364ac7aae6de12dbe858c8"
          }
        },
        "2a832f046e554c00acbc7559f6104550": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4746c7989454b588dcdbe4aa347593d",
            "placeholder": "​",
            "style": "IPY_MODEL_18b0d4c398c346f78edd84926699c86d",
            "value": "Generating validation split:   0%"
          }
        },
        "b4c146f0f0194dc99f8d28ea3ae01719": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7383f1b50747460db4fb2a841aaac9e7",
            "max": 888,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1979b206b4d84176ab7e2e41f4870deb",
            "value": 888
          }
        },
        "ce0af431d90047ea9ba4255809220a5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_14925890f80a43e6a125e9957e509228",
            "placeholder": "​",
            "style": "IPY_MODEL_b63a5e406bfe4cec8abff8fd31938fb5",
            "value": " 0/888 [00:00&lt;?, ? examples/s]"
          }
        },
        "c4cc841254364ac7aae6de12dbe858c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "e4746c7989454b588dcdbe4aa347593d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18b0d4c398c346f78edd84926699c86d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7383f1b50747460db4fb2a841aaac9e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1979b206b4d84176ab7e2e41f4870deb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "14925890f80a43e6a125e9957e509228": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b63a5e406bfe4cec8abff8fd31938fb5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2ec6b9ce358c45f883ec9b7e93d4bd21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2b4132f308274e2fa64e70383632c348",
              "IPY_MODEL_a7ebead26e2a4ea19ccc3c10a06413a4",
              "IPY_MODEL_f24d28b7c2da47cf84f206f14335072f"
            ],
            "layout": "IPY_MODEL_897783f78ead41408457716c8d426e19"
          }
        },
        "2b4132f308274e2fa64e70383632c348": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6cd672d1d49b4780881b6483e7f22763",
            "placeholder": "​",
            "style": "IPY_MODEL_01946d4ce86b4a20aed3ed599962aa11",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "a7ebead26e2a4ea19ccc3c10a06413a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e0652c6130864cdc955f76c856eab887",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0a778f305d214b77990ee3927edb6bbe",
            "value": 466062
          }
        },
        "f24d28b7c2da47cf84f206f14335072f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e9594efa6d04144829116ba877838ce",
            "placeholder": "​",
            "style": "IPY_MODEL_37b6e138f0714db0bb8efc4a7b475217",
            "value": " 466k/466k [00:00&lt;00:00, 6.51MB/s]"
          }
        },
        "897783f78ead41408457716c8d426e19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6cd672d1d49b4780881b6483e7f22763": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01946d4ce86b4a20aed3ed599962aa11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e0652c6130864cdc955f76c856eab887": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a778f305d214b77990ee3927edb6bbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3e9594efa6d04144829116ba877838ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37b6e138f0714db0bb8efc4a7b475217": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c8df6ae771c14d25a0938f025b8fe28d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d0d8a3e2968b4836b06b00932d214c52",
              "IPY_MODEL_f7f9c83502204976835843f7921cf382",
              "IPY_MODEL_817c7b2032f14ea6bed78fb502ed22ee"
            ],
            "layout": "IPY_MODEL_b3e3b6542744446987737c97f5294db0"
          }
        },
        "d0d8a3e2968b4836b06b00932d214c52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d855971788dc4d6eb501e84f146acbad",
            "placeholder": "​",
            "style": "IPY_MODEL_cfa49ee604e54333a1e41d2c8800fd6e",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "f7f9c83502204976835843f7921cf382": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f69ac5e723f94865b0b60bb191acfc91",
            "max": 29,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4436339f386a4bc2a657131382b0deae",
            "value": 29
          }
        },
        "817c7b2032f14ea6bed78fb502ed22ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0bd4b6aaf48426ba62f6f42c9465cdc",
            "placeholder": "​",
            "style": "IPY_MODEL_66dd3b97c7bd43cb85b62894f9c6451b",
            "value": " 29.0/29.0 [00:00&lt;00:00, 1.45kB/s]"
          }
        },
        "b3e3b6542744446987737c97f5294db0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d855971788dc4d6eb501e84f146acbad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfa49ee604e54333a1e41d2c8800fd6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f69ac5e723f94865b0b60bb191acfc91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4436339f386a4bc2a657131382b0deae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b0bd4b6aaf48426ba62f6f42c9465cdc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66dd3b97c7bd43cb85b62894f9c6451b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f83863fcd03644088489bf1e37db18df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c80756835d7c4d4ba7d317c231ff1458",
              "IPY_MODEL_6d0be7b97a144a199f869723f601cb4e",
              "IPY_MODEL_d0061ae7f41c473382bf8530a070d8f0"
            ],
            "layout": "IPY_MODEL_f1e75741265d4fb7b3456ad59de108fd"
          }
        },
        "c80756835d7c4d4ba7d317c231ff1458": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_668df119ec6e4d129c7bd9feb5ffba9b",
            "placeholder": "​",
            "style": "IPY_MODEL_08e2b3be0cc049a69ef1f23c56336761",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "6d0be7b97a144a199f869723f601cb4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b6bf47ad411842fb94f63864d7854e40",
            "max": 433,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2ef31812ea784731b9e060ade9f5bd79",
            "value": 433
          }
        },
        "d0061ae7f41c473382bf8530a070d8f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1ef87883ceb8496486a41b77e28d5824",
            "placeholder": "​",
            "style": "IPY_MODEL_128aa88c81b6466ba773d7d0de90b93b",
            "value": " 433/433 [00:00&lt;00:00, 28.4kB/s]"
          }
        },
        "f1e75741265d4fb7b3456ad59de108fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "668df119ec6e4d129c7bd9feb5ffba9b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08e2b3be0cc049a69ef1f23c56336761": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b6bf47ad411842fb94f63864d7854e40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2ef31812ea784731b9e060ade9f5bd79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1ef87883ceb8496486a41b77e28d5824": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "128aa88c81b6466ba773d7d0de90b93b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ffcd03a5d5e748feae7293e291daa399": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c4e214bc2e91443b8ac5cde5de97c704",
              "IPY_MODEL_cbdc17b29ad1430ebdaf92e3adb61434",
              "IPY_MODEL_e92947565c4d412fad22038072c3a7b6"
            ],
            "layout": "IPY_MODEL_c073b9e046394f0998460e50939f2333"
          }
        },
        "c4e214bc2e91443b8ac5cde5de97c704": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_45d1ed8ec10e4a7eaab365e5b273ddf9",
            "placeholder": "​",
            "style": "IPY_MODEL_262b879df80b493787970f603f50388f",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "cbdc17b29ad1430ebdaf92e3adb61434": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5faccde4837c4fcab7b750e39a6a0c87",
            "max": 254728,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8c7bb2f1888146cb85c059145d13aebf",
            "value": 254728
          }
        },
        "e92947565c4d412fad22038072c3a7b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_36825e58c18445e3a2094f165f7d32dc",
            "placeholder": "​",
            "style": "IPY_MODEL_a1eeb4106ab7423a8f5eddd086f17a7c",
            "value": " 255k/255k [00:00&lt;00:00, 5.99MB/s]"
          }
        },
        "c073b9e046394f0998460e50939f2333": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45d1ed8ec10e4a7eaab365e5b273ddf9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "262b879df80b493787970f603f50388f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5faccde4837c4fcab7b750e39a6a0c87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c7bb2f1888146cb85c059145d13aebf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "36825e58c18445e3a2094f165f7d32dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a1eeb4106ab7423a8f5eddd086f17a7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8e40600986ac49bba5ce7e23d35c341a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_72f6b4cbac20482db0ec36951dce4cd9",
              "IPY_MODEL_095038d110b04b88bd0f0ec19f333dc4",
              "IPY_MODEL_f569ed617bd94d92a909d8c83d63c7b2"
            ],
            "layout": "IPY_MODEL_312071c63dc54e2284c14b618260bade"
          }
        },
        "72f6b4cbac20482db0ec36951dce4cd9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8e8d2973aa6345c98e58e0f5cd73f69c",
            "placeholder": "​",
            "style": "IPY_MODEL_3d1d1b34f905455ab86f836fc3f6aef3",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "095038d110b04b88bd0f0ec19f333dc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_15011dd37b344df3ba8af0cce04dc348",
            "max": 485115,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1b0259af9b0f492eb1f70ce56994e80a",
            "value": 485115
          }
        },
        "f569ed617bd94d92a909d8c83d63c7b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12399f28e4fe4bfc83307fd4481dacfa",
            "placeholder": "​",
            "style": "IPY_MODEL_59f2fa9d24b944379951ed4920437ba6",
            "value": " 485k/485k [00:00&lt;00:00, 7.29MB/s]"
          }
        },
        "312071c63dc54e2284c14b618260bade": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e8d2973aa6345c98e58e0f5cd73f69c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d1d1b34f905455ab86f836fc3f6aef3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "15011dd37b344df3ba8af0cce04dc348": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b0259af9b0f492eb1f70ce56994e80a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "12399f28e4fe4bfc83307fd4481dacfa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "59f2fa9d24b944379951ed4920437ba6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Text Classification"
      ],
      "metadata": {
        "id": "asjpx0fjgKnd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GAelwZcJdw4N"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_device():\n",
        "  device=\"cpu\"\n",
        "  if torch.cuda.is_available():\n",
        "    device=\"cuda\"\n",
        "  elif  torch.backends.mps.is_available():\n",
        "    device='mps'\n",
        "  else:\n",
        "    device=\"cpu\"\n",
        "  return device\n",
        "\n",
        "\n",
        "device = get_device()\n",
        "print(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QnqyNd3HtnK3",
        "outputId": "3ef11595-6945-463e-df24-b11bf2579b14"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mps\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self, dim_embedding, dropout=0.1, max_seq_len=5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "        postional_encoding = torch.zeros(max_seq_len, dim_embedding)\n",
        "        position = torch.arange(0, max_seq_len, dtype=torch.float).unsqueeze(1)\n",
        "        denom_term = torch.exp(torch.arange(0, dim_embedding, 2).float() * (-math.log(10000.0) / dim_embedding))\n",
        "        postional_encoding[:, 0::2] = torch.sin(position * denom_term)\n",
        "        postional_encoding[:, 1::2] = torch.cos(position * denom_term)\n",
        "        postional_encoding = postional_encoding.unsqueeze(0).transpose(0, 1)\n",
        "        self.register_buffer('postional_encoding', postional_encoding)\n",
        "    def forward(self, x):\n",
        "        x = x + self.postional_encoding[:x.size(0), :]\n",
        "        return self.dropout(x)"
      ],
      "metadata": {
        "id": "8nIbQdvwlrfk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Preparation: \n",
        "Here's our plan for data preparation.\n",
        "Prepare the IMDb dataset for text classification:\n",
        "\n",
        "1. Load the IMDb dataset and tokenize the text using an appropriate pre-trained tokenizer.\n",
        "2.Pad and truncate the tokenized text to ensure a consistent sequence length across different examples.\n",
        "3. Ensure the data shapes are compatible with the TransformerEncoder requirements:\n",
        "  * Reshape the tokenized data (input_ids) to have a shape of (seq_length, batch_size).\n",
        "  * Reshape the attention mask to have a shape of (batch_size, seq_length).\n",
        "  * One-hot encode the labels for binary-class classification.\n",
        "\n",
        "4. Create TensorDataset to use the huggingface data on Pytorch.\n",
        "  * TensorDataset is a utility class in PyTorch (from the torch.utils.data module) that allows you to create a dataset object by wrapping one or more tensors. Each tensor in the dataset represents a different field or attribute of your data samples, such as input data, labels, or attention masks.\n",
        "5. Define custom Collate_fn\n",
        "  * The default collate function in DataLoader simply combines the samples into a batch without any additional processing. However, in many cases, you may need to perform custom processing on your samples\n",
        "  * Accorind to our shape requirement. Performe Shape transformation in this function.\n",
        "  * data: (seq_len, batch_size)\n",
        "  * atten_mask: (batch_size, seq_len)\n",
        "  * labe: one-hot-encode\n",
        "\n",
        "Important Point: Understand the Attention Mask:\n",
        "\n",
        "  * The attention mask is a binary tensor that indicates which tokens in the input sequence should be attended to by the model.\n",
        "In the attention mask tensor, the value 1 corresponds to an actual token (word or subword), while the value 0 corresponds to a padding token.\n",
        "  * The model uses this mask to ignore padding tokens during the self-attention mechanism in the Transformer architecture.\n",
        "\n",
        "Let's Code it!"
      ],
      "metadata": {
        "id": "kRGC2oBuEZXr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "from transformers import AutoTokenizer\n",
        "from torch.utils.data import TensorDataset\n",
        "\n",
        "\n",
        "# Load the dataset and tokenizer\n",
        "dataset = load_dataset(\"imdb\")\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
        "\n",
        "# Tokenize the dataset\n",
        "def tokenize(batch):\n",
        "    return tokenizer(batch[\"text\"], padding=True, truncation=True, return_tensors=\"pt\", max_length=512)\n",
        "\n",
        "train_dataset = dataset[\"train\"].map(tokenize, batched=True, batch_size=len(dataset[\"train\"]))\n",
        "val_dataset = dataset[\"test\"].map(tokenize, batched=True, batch_size=len(dataset[\"test\"]))\n",
        "\n",
        "# Extract input_ids and attention_mask from the tokenized dataset\n",
        "train_data = torch.tensor(train_dataset[\"input_ids\"])\n",
        "train_attention_mask = torch.tensor(train_dataset[\"attention_mask\"])\n",
        "train_labels = torch.tensor(train_dataset[\"label\"])\n",
        "\n",
        "val_data = torch.tensor(val_dataset[\"input_ids\"])\n",
        "val_attention_mask = torch.tensor(val_dataset[\"attention_mask\"])\n",
        "val_labels = torch.tensor(val_dataset[\"label\"])\n",
        "\n",
        "# Create TensorDatasets\n",
        "train_dataset = TensorDataset(train_data, train_attention_mask, train_labels)\n",
        "val_dataset = TensorDataset(val_data, val_attention_mask, val_labels)\n",
        "\n",
        "# Create DataLoaders\n",
        "\n",
        "def collate_fn(batch):\n",
        "    input_ids, attention_mask, labels = zip(*batch)\n",
        "    input_ids = torch.stack(input_ids).transpose(0, 1)  # Transpose input_ids\n",
        "    attention_mask = torch.stack(attention_mask)   # Transpose attention_mask\n",
        "    labels = torch.nn.functional.one_hot(torch.tensor(labels), num_classes=2).float().to(device)\n",
        "    return input_ids, attention_mask, labels\n",
        "\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True, collate_fn=collate_fn)\n",
        "val_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False, collate_fn=collate_fn)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 123,
          "referenced_widgets": [
            "2df0a1f692e64fdfb0cefe79815f0b72",
            "f6f6fd67d3ba434c967461f8ccaf0d8e",
            "1803368fc8644332b7be4b4ca60bb2a8",
            "eebf7c12549f4abb96a514038f678bbf",
            "5ecef016b03f4887ae6711875752b77f",
            "c4895a530b424d18a5788d39dbb5e286",
            "8d3775b9b66a44ff8e419186e66560f2",
            "ebd185a979a04d639b0d98656ad6ee39",
            "bc0604af87f947639663a3641173a29c",
            "7b2d74d09b104c7bb536fba61289a556",
            "4d1e69159a674720ab6c059a93e5b9d3"
          ]
        },
        "id": "O5X0BwHAqc4-",
        "outputId": "9cfb3a52-ceba-4d69-e783-1782de02db40"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Found cached dataset imdb (/Users/premtimsina/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2df0a1f692e64fdfb0cefe79815f0b72"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /Users/premtimsina/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0/cache-5ff46743b60fe77a.arrow\n",
            "Loading cached processed dataset at /Users/premtimsina/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0/cache-b5dbdc1fc6b34a20.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The src_key_padding_mask in the TransformerEncoder should be a 2D boolean tensor of shape (batch_size, sequence_length). Each element in the tensor should be True if the corresponding token in the input sequence is a padding token, and False otherwise.\n",
        "\n",
        "The src_key_padding_mask is used to mask out the padding tokens in the input so that they don't contribute to the attention calculation. When the padding tokens are masked out, the transformer will not consider them while computing attention scores for the non-padding tokens."
      ],
      "metadata": {
        "id": "BD0ZNbfj3AZJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "This is our idea for Text Classification\n",
        "1. Initialize the following components in the constructor:\n",
        "  * Embedding layer to convert input tokens into embeddings.\n",
        "  * Positional encoding to add position information to the embeddings.\n",
        "  * Transformer encoder consisting of multiple layers, each with multi-head self-attention and feedforward neural networks.\n",
        "  * Fully connected (linear) layer for classification.\n",
        "2. Implement the init_weights method to initialize the weights of the model components.\n",
        "\n",
        "3. Implement the forward method to define the forward pass of the model.\n",
        "  * Pass the input through the embedding layer and apply positional encoding. We also add multiplication term to the embedding. The purpose of this multiplication is to scale the embeddings. This can help the model learn better and avoid vanishing gradients. The square root of the model's dimension is used as a scaling factor because it is a simple heuristic that works well in practice.\n",
        "  * Pass the embeddings through the transformer encoder with an optional key_padding_mask.\n",
        "  * Perform mean pooling on the last dimension and use the first token representation.\n",
        "    1. The output of encoder is (Seq_length, batch_size, emb_dim). Performing the mean polling across the dimesion `0` will change the data in the form of (batch_size, emb_dim)\n",
        "  * Pass the pooled representation through the fully connected layer for classification.\n",
        "  * Apply a sigmoid activation function to obtain probabilities."
      ],
      "metadata": {
        "id": "H2j5LTHgLp6S"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class TextClassifier(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_dim, nhead, num_layers, num_classes):\n",
        "        super(TextClassifier, self).__init__()\n",
        "\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.positional_encoding = PositionalEncoding(embedding_dim)\n",
        "        # Create the transformer encoder layer\n",
        "        self.encoder_layer = nn.TransformerEncoderLayer(embedding_dim, nhead)\n",
        "        self.encoder = nn.TransformerEncoder(self.encoder_layer, num_layers)\n",
        "        self.fc = nn.Linear(embedding_dim, num_classes)\n",
        "        self.embedding_dim=embedding_dim\n",
        "        self.init_weights()\n",
        "\n",
        "    def init_weights(self) -> None:\n",
        "        initrange = 0.1\n",
        "        self.embedding.weight.data.uniform_(-initrange, initrange)\n",
        "        for layer in self.encoder.layers:\n",
        "            nn.init.xavier_uniform_(layer.self_attn.out_proj.weight)\n",
        "            nn.init.zeros_(layer.self_attn.out_proj.bias)\n",
        "            nn.init.xavier_uniform_(layer.linear1.weight)\n",
        "            nn.init.zeros_(layer.linear1.bias)\n",
        "            nn.init.xavier_uniform_(layer.linear2.weight)\n",
        "            nn.init.zeros_(layer.linear2.bias)\n",
        "        self.fc.bias.data.zero_()\n",
        "        self.fc.weight.data.uniform_(-initrange, initrange)\n",
        "\n",
        "    def forward(self, x, key_padding_mask=None):\n",
        "        x = self.embedding(x)* math.sqrt(self.embedding_dim)\n",
        "        x = self.positional_encoding(x)\n",
        "        x = self.encoder(x, src_key_padding_mask=key_padding_mask)\n",
        "\n",
        "        # Pooling the last dimension and use the first token representation\n",
        "        x = x.mean(dim=0)\n",
        "\n",
        "        # Fully connected layer for classification\n",
        "        x = self.fc(x)\n",
        "        x=torch.sigmoid(x)\n",
        "        return x"
      ],
      "metadata": {
        "id": "hk5CadiUsiL_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Declaring Model"
      ],
      "metadata": {
        "id": "_h9HFIpET-a3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vocab_size = tokenizer.vocab_size\n",
        "embedding_dim = 512\n",
        "nhead = 8\n",
        "num_layers = 6\n",
        "num_classes = 2\n",
        "\n",
        "# Create the model\n",
        "model = TextClassifier(vocab_size, embedding_dim, nhead, num_layers,  num_classes).to(device)\n",
        "criterion = nn.BCELoss().to(device)\n",
        "optimizer = optim.Adam(model.parameters(), lr=0.001)"
      ],
      "metadata": {
        "id": "yB7kz5AxtLQ3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training\n",
        "Here are a few important things to note:\n",
        "\n",
        "* batch_attention_mask = (batch_attention_mask == 0).to(device): Convert the attention mask to a boolean tensor by checking if the values are equal to 0.\n",
        "  1. The attention mask that comes from Hugging Face tokenizer is as follows:\n",
        "    * attention_mask = 1 if it is a real token\n",
        "    * attention_mask = 0 if it is a pad token\n",
        "  2. PyTorch's attention_mask requires us to send:\n",
        "    * attention_mask = False if it is a real token\n",
        "    * attention_mask = True if it is a pad token\n",
        "  3. Therefore, we are converting batch_attention_mask == 0 to True. This means we are telling the model that if it is a pad token, set attention_mask = True.\n",
        "* To avoid exploding gradients, clip the gradients of the model's parameters using torch.nn.utils.clip_grad_norm_(). This helps maintain the stability of the training process and prevents the gradients from becoming too large.\n"
      ],
      "metadata": {
        "id": "yfDDSZy1kOcL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "num_epochs = 1\n",
        "for epoch in range(num_epochs):\n",
        "    i=0\n",
        "    for batch_data, batch_attention_mask, batch_labels in train_dataloader:\n",
        "      \n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # Convert attention_mask to boolean tensor\n",
        "        batch_attention_mask = (batch_attention_mask==0).to(device)\n",
        "\n",
        "        outputs = model(batch_data.to(device), key_padding_mask=batch_attention_mask)\n",
        "        loss = criterion(outputs, batch_labels.to(device))\n",
        "        if i%100==0:\n",
        "          print (\"epoch \", epoch, \"batch \", i, \"loss \", loss)\n",
        "\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n",
        "        optimizer.step()\n",
        "        i=i+1\n",
        "\n",
        "    print(f\"Epoch: {epoch + 1}, Loss: {loss.item()}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MhkFxM_gs3uo",
        "outputId": "9f0ddb57-da8d-4987-cb1c-6415a9fe89e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch  0 batch  0 loss  tensor(0.8329, device='mps:0', grad_fn=<BinaryCrossEntropyBackward0>)\n",
            "epoch  0 batch  100 loss  tensor(0.7013, device='mps:0', grad_fn=<BinaryCrossEntropyBackward0>)\n",
            "epoch  0 batch  200 loss  tensor(0.6965, device='mps:0', grad_fn=<BinaryCrossEntropyBackward0>)\n",
            "epoch  0 batch  300 loss  tensor(0.6912, device='mps:0', grad_fn=<BinaryCrossEntropyBackward0>)\n",
            "epoch  0 batch  400 loss  tensor(0.6935, device='mps:0', grad_fn=<BinaryCrossEntropyBackward0>)\n",
            "epoch  0 batch  500 loss  tensor(0.6960, device='mps:0', grad_fn=<BinaryCrossEntropyBackward0>)\n",
            "epoch  0 batch  600 loss  tensor(0.6958, device='mps:0', grad_fn=<BinaryCrossEntropyBackward0>)\n",
            "epoch  0 batch  700 loss  tensor(0.6929, device='mps:0', grad_fn=<BinaryCrossEntropyBackward0>)\n",
            "Epoch: 1, Loss: 0.6728882789611816\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model.state_dict(), \"/Users/premtimsina/Documents/bpbbook/chapter3/TextClassificationModel.pth\")"
      ],
      "metadata": {
        "id": "ijB4RY6I6WBS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Inference\n"
      ],
      "metadata": {
        "id": "GeLxaeA4mI8o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Instantiate the tokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
        "vocab_size = tokenizer.vocab_size\n",
        "embedding_dim = 512\n",
        "nhead = 8\n",
        "num_layers = 3\n",
        "num_classes = 2\n",
        "\n",
        "# Create the model\n",
        "model_loaded = TextClassifier(vocab_size, embedding_dim, nhead, num_layers,  num_classes).to(device)\n",
        "\n",
        "# Load the trained model weights (replace 'path_to_weights.pth' with the path to your trained model)\n",
        "model_loaded.load_state_dict(torch.load('/Users/premtimsina/Documents/bpbbook/chapter3/TextClassificationModel.pth'))\n",
        "model.eval()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NSFIm_lH_kCf",
        "outputId": "0c9d9f5e-3d1e-4aeb-ec3f-748301cfc002"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TextClassifier(\n",
              "  (embedding): Embedding(30522, 512)\n",
              "  (positional_encoding): PositionalEncoding(\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (encoder_layer): TransformerEncoderLayer(\n",
              "    (self_attn): MultiheadAttention(\n",
              "      (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
              "    )\n",
              "    (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "    (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "    (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "    (dropout1): Dropout(p=0.1, inplace=False)\n",
              "    (dropout2): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (encoder): TransformerEncoder(\n",
              "    (layers): ModuleList(\n",
              "      (0-2): 3 x TransformerEncoderLayer(\n",
              "        (self_attn): MultiheadAttention(\n",
              "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "        (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "        (dropout1): Dropout(p=0.1, inplace=False)\n",
              "        (dropout2): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (fc): Linear(in_features=512, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 184
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Printing total parameter of our model"
      ],
      "metadata": {
        "id": "FZQSr0QImPDA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "print(\"Total trainable parameters:\", total_params)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L8ZAWHdjOn9r",
        "outputId": "7c71420d-c976-45db-afce-8cad9ec584dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total trainable parameters: 37694978\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Function to perform inference on a given text\n",
        "def infer(text):\n",
        "    # Tokenize the input text\n",
        "    tokens = tokenizer.encode_plus(text, padding=True, truncation=True, return_tensors=\"pt\", max_length=512)\n",
        "    input_ids = tokens[\"input_ids\"].to(device).transpose(0,1)\n",
        "\n",
        "    attention_mask = tokens[\"attention_mask\"]\n",
        "    attention_mask=(attention_mask==0).to(device)\n",
        "    print(input_ids.shape)\n",
        "    print(attention_mask)\n",
        "\n",
        "    # Perform inference\n",
        "    with torch.no_grad():\n",
        "        output = model(input_ids, key_padding_mask=attention_mask)\n",
        "    # Convert the output to class probabilities\n",
        "    probabilities = output.squeeze(0)\n",
        "    return probabilities\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "CrNKSUlp6bmS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Test with an example text\n",
        "example_text = \"This movie is  good! .\"\n",
        "probabilities = infer(example_text)\n",
        "\n",
        "print(\"Probabilities:\", probabilities)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_gavMpQn9B9A",
        "outputId": "5f88bba5-a0a7-4c28-dc36-28474c1ac2ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([8, 1])\n",
            "tensor([[False, False, False, False, False, False, False, False]],\n",
            "       device='mps:0')\n",
            "Probabilities: tensor([0.4503, 0.5445], device='mps:0')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Summary:\n",
        "The provided code offers a basic implementation of a TransformerEncoder for text classification. The primary goal is to demonstrate how to utilize a TransformerEncoder for this task. In subsequent chapters, we will explore optimized versions of Transformer-based classification. To achieve better results with the current code, consider increasing the number of encoder layers and fine-tuning various parameters."
      ],
      "metadata": {
        "id": "MdF9gaqdMSEY"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Text Generation\n",
        "## Decoder-Only Layer\n",
        "1. We will use the Shakespeare dataset to create a decoder-only model that generates text in the style of Shakespeare."
      ],
      "metadata": {
        "id": "T2F5wPqSPH-B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n"
      ],
      "metadata": {
        "id": "fnop0O8VB-ia"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "What we are doing Here?\n",
        "1. This code defines a custom PyTorch dataset called ShakespeareDataset.\n",
        "2. It takes a file path to a text file and a tokenizer as input. The dataset reads the file and splits it into examples of size block_size. \n",
        "3. Each example is then tokenized using the tokenizer and padded or truncated to a maximum length of block_size. The resulting tokenized examples are stored in a list."
      ],
      "metadata": {
        "id": "Q-0Oowr8K9hT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class ShakespeareDataset(Dataset):\n",
        "    def __init__(self, file_path, tokenizer, block_size=128):\n",
        "        self.block_size = block_size\n",
        "        self.tokenizer = tokenizer\n",
        "\n",
        "        with open(file_path, 'r') as f:\n",
        "            self.data = f.read()\n",
        "\n",
        "        self.examples = []\n",
        "        for i in range(0, len(self.data)-self.block_size, self.block_size):\n",
        "            example = self.data[i:i+self.block_size]\n",
        "            tokenized = self.tokenizer(example, padding='max_length', truncation=True, max_length=block_size, return_tensors='pt')\n",
        "            self.examples.append(tokenized)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.examples)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        input_ids = self.examples[idx]['input_ids'].squeeze()\n",
        "        attention_mask = self.examples[idx]['attention_mask'].squeeze()\n",
        "        return input_ids, attention_mask\n",
        "\n",
        "  \n"
      ],
      "metadata": {
        "id": "hmi7tWV5CHG1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os,urllib\n",
        "url = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt'\n",
        "filename = '/Users/premtimsina/Documents/bpbbook/chapter3/dataset/input.txt'\n",
        "if not os.path.isfile(filename):\n",
        "    urllib.request.urlretrieve(url, filename)\n"
      ],
      "metadata": {
        "id": "ydFhPczFCNAU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = ShakespeareDataset(filename, tokenizer)"
      ],
      "metadata": {
        "id": "7IGiDxcFC6ev"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "What we are doing here?\n",
        "1. `inputs = torch.stack(inputs).transpose(0, 1)`: The dimension of input needed is (seq_length, batch_size). Thus, we are transposing.\n",
        "2. `torch.stack(inputs)` takes a list of tensors and stacks them along a new dimension (the result has one more dimension than the input tensors). \n",
        "  * For example, each inputs has dimension of (128). If the batch size 6; then, stacking operation will produce list of (6,128)=> (batch_size, seq_length)\n",
        "  * transpose will result in (128,6)==> (seq_length, batch_size)"
      ],
      "metadata": {
        "id": "7Thc88UNLYZU"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "BmWsAziDLVwr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.nn.utils.rnn import pad_sequence\n",
        "def collate_fn(batch):\n",
        "    inputs, masks = zip(*batch)\n",
        "    inputs = torch.stack(inputs).transpose(0, 1)\n",
        "    masks = torch.stack(masks)\n",
        "    return inputs, masks\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=4, collate_fn=collate_fn,shuffle=True)\n"
      ],
      "metadata": {
        "id": "Fz0fsqDeDx48"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Just looking at the dimension of data\n",
        "item=next(iter(train_dataloader))\n",
        "input_ids,attention_masks=item\n",
        "print(input_ids.shape, attention_masks.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WmZMRIs-F-Ri",
        "outputId": "146fa47b-68e0-4bc8-987b-5c8b315f683a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([128, 4]) torch.Size([4, 128])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Let's Declare Positional Encoding Class.\n",
        "1. It is same as what we discussed in earlier example"
      ],
      "metadata": {
        "id": "oXL3V8RXM3Dm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self, dim_embedding, dropout=0.1, max_seq_len=5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "        postional_encoding = torch.zeros(max_seq_len, dim_embedding)\n",
        "        position = torch.arange(0, max_seq_len, dtype=torch.float).unsqueeze(1)\n",
        "        denom_term = torch.exp(torch.arange(0, dim_embedding, 2).float() * (-math.log(10000.0) / dim_embedding))\n",
        "        postional_encoding[:, 0::2] = torch.sin(position * denom_term)\n",
        "        postional_encoding[:, 1::2] = torch.cos(position * denom_term)\n",
        "        postional_encoding = postional_encoding.unsqueeze(0).transpose(0, 1)\n",
        "        self.register_buffer('postional_encoding', postional_encoding)\n",
        "    def forward(self, x):\n",
        "        x = x + self.postional_encoding[:x.size(0), :]\n",
        "        return self.dropout(x)"
      ],
      "metadata": {
        "id": "m99atpU2KFw9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Let's Declare Model\n",
        "### Model Architecture\n",
        "1. This model is a Transformer-based decoder-only language model, which takes as input a target sequence (tgt) and an  memory sequence (memory) and generates an output sequence of the same length as the input sequence.\n",
        "\n",
        "2. The input target sequence is first passed through an embedding layer and a positional encoding layer. Similarly, the input memory sequence is passed through an embedding layer and a positional encoding layer.\n",
        "\n",
        "3. During, Training\n",
        "  * `memory` is train data of shape (seq_len, batch_size)\n",
        "  * `target`:During model training, the target sequence would be the input sequence shifted by one position.\n",
        "\n",
        "4. These processed input sequences are then fed into the Transformer decoder, which consists of multiple Transformer decoder layers. Each decoder layer processes the input sequences using multi-head self-attention and a feedforward neural network.\n",
        "\n",
        "5. Finally, the output of the Transformer decoder is passed through a linear layer (fully-connected neural network) to generate the final output sequence, with each element of the sequence representing the probability distribution over the vocabulary of the target language."
      ],
      "metadata": {
        "id": "ujk42ZNKNaMA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class TransformerDecoder(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_dim, num_layers, dropout):\n",
        "        super().__init__()\n",
        "\n",
        "        self.memory_embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.memory_pos_encoder = PositionalEncoding(embedding_dim, dropout)\n",
        "        self.tgt_embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.tgt_pos_encoder = PositionalEncoding(embedding_dim, dropout)\n",
        "        self.decoder = nn.TransformerDecoder(\n",
        "            nn.TransformerDecoderLayer(d_model=embedding_dim, nhead=8, dim_feedforward=2048, dropout=dropout),\n",
        "            num_layers=num_layers)\n",
        "\n",
        "        self.fc = nn.Linear(embedding_dim, vocab_size)\n",
        "        self.d_model=embedding_dim\n",
        "        self.init_weights()\n",
        "\n",
        "    def init_weights(self) -> None:\n",
        "        initrange = 0.1\n",
        "\n",
        "        # Initialize the embedding layers\n",
        "        nn.init.uniform_(self.memory_embedding.weight, -initrange, initrange)\n",
        "        nn.init.uniform_(self.tgt_embedding.weight, -initrange, initrange)\n",
        "\n",
        "        # Initialize the decoder layers\n",
        "        for param in self.decoder.parameters():\n",
        "            if param.dim() > 1:\n",
        "                nn.init.xavier_uniform_(param)\n",
        "\n",
        "        # Initialize the output layer\n",
        "        nn.init.uniform_(self.fc.weight, -initrange, initrange)\n",
        "        nn.init.zeros_(self.fc.bias)\n",
        "    def forward(self, tgt,  memory=None, tgt_mask=None, memory_mask=None, memory_key_padding_mask=None,tgt_key_padding_mask=None):\n",
        "        tgt = self.tgt_embedding(tgt) * self.d_model ** 0.5\n",
        "        tgt=self.tgt_pos_encoder(tgt)\n",
        "        print(tgt)\n",
        "        memory=self.memory_embedding(memory) * self.d_model ** 0.5\n",
        "        memory=self.memory_pos_encoder(memory)\n",
        "        print(memory)\n",
        "        output = self.decoder(tgt=tgt, memory=memory, tgt_mask=tgt_mask, memory_mask=memory_mask, memory_key_padding_mask=memory_key_padding_mask,tgt_key_padding_mask=tgt_key_padding_mask)\n",
        "        print(output)\n",
        "        output = self.fc(output)\n",
        "        return output"
      ],
      "metadata": {
        "id": "kxVdZrgWD4bc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "eS4oE3_RlkcW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Parameters  for decoding Layer\n",
        "\n",
        "1. tgt: The input sequence to the decoder layer. It is a tensor of shape (seq_len, batch_size, emb_dim) where seq_len is the length of the input sequence and batch_size is the number of sequences in a batch.\n",
        "\n",
        "2. memory: The output of the last layer of the encoder. It is a tensor of shape (src_seq_len, batch_size, emb_dim) where src_seq_len is the length of the input sequence in the encoder.\n",
        "\n",
        "3. tgt_mask: An optional tensor of shape (seq_len, seq_len) representing the mask for the input sequence. It is used to prevent the decoder from attending to future tokens.\n",
        "The format should be: \n",
        "```\n",
        "tensor([[0., -inf, -inf],\n",
        "        [0., 0., -inf],\n",
        "        [0., 0., 0.]], device='mps:0')\n",
        "```\n",
        "  * in above example, seq_length=3\n",
        "  * where `-inf` signifies the tokens that need to be masked\n",
        "\n",
        "4. memory_mask: An optional tensor of shape (seq_len, src_seq_len) representing the mask for the encoder output sequence. It is used to prevent the decoder from attending future tokens in the encoder input sequence.\n",
        "```\n",
        "tensor([[0., -inf, -inf],\n",
        "        [0., 0., -inf],\n",
        "        [0., 0., 0.]], device='mps:0')\n",
        "```\n",
        "  * in above example, seq_length=3\n",
        "  * where `-inf` signifies the tokens that need to be masked\n",
        "```\n",
        "Usually, you will not mask the memory: Thus, you will pass:\n",
        "tensor([[0., 0, 0],\n",
        "        [0., 0., 0],\n",
        "        [0., 0., 0.]], device='mps:0')\n",
        "\n",
        "5. tgt_key_padding_mask: An optional tensor of shape (batch_size, seq_len) representing the mask for padding tokens in the input sequence.\n",
        "```\n",
        "tensor([[False, False, False],\n",
        "        [False, False, False],\n",
        "        [False, True, False],\n",
        "        [True, True, False]], device='mps:0')\n",
        "```\n",
        "  * In above example, batch_size=4,  seq_len=3\n",
        "  * True signifies the particular token is padded token and mask it\n",
        "  * False signifies the particular token is padded token and mask it\n",
        "\n",
        "6. memory_key_padding_mask: An optional tensor of shape (batch_size, src_seq_len) representing the mask for padding tokens in the encoder output sequence.\n",
        "```\n",
        "tensor([[False, False, False],\n",
        "        [False, False, False],\n",
        "        [False, True, False],\n",
        "        [True, True, False]], device='mps:0')\n",
        "```\n",
        "  * In above example, batch_size=4,  seq_len=3\n",
        "  * True signifies the particular token is padded token and mask it\n",
        "  * False signifies the particular token is padded token and mask it\n"
      ],
      "metadata": {
        "id": "IIAcfLAYdUjV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_square_subsequent_mask(sz):\n",
        "    mask = (torch.triu(torch.ones((sz, sz), device=device)) == 1).transpose(0, 1)\n",
        "    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
        "    return mask.to(device)\n",
        "\n",
        "\n",
        "def create_mask(src, tgt,tokenizer_src=tokenizer,tokenizer_tgt=tokenizer):\n",
        "    src_seq_len = src.shape[0]\n",
        "    tgt_seq_len = tgt.shape[0]\n",
        "\n",
        "    tgt_mask = generate_square_subsequent_mask(tgt_seq_len)\n",
        "    src_mask = torch.zeros((src_seq_len, src_seq_len),device=device).type(torch.bool)\n",
        "\n",
        "    src_padding_mask = (src == tokenizer_src.pad_token_id).transpose(0, 1)\n",
        "    tgt_padding_mask = (tgt == tokenizer_tgt.pad_token_id).transpose(0, 1)\n",
        "    return src_mask.to(device), tgt_mask.to(device), src_padding_mask.to(device), tgt_padding_mask.to(device)"
      ],
      "metadata": {
        "id": "xILYGE1JHYt9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import math\n",
        "device='mps'\n",
        "model = TransformerDecoder(vocab_size=tokenizer.vocab_size, embedding_dim=768, num_layers=3, dropout=0.1).to(device)\n",
        "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
        "criterion = nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id)\n"
      ],
      "metadata": {
        "id": "wtUNadG0D_xb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Exercise \n",
        "1. Write the Training Loop\n",
        "2. Write the Inference Loop"
      ],
      "metadata": {
        "id": "2wMH1JTiB87y"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Transformer Layer\n",
        "## Machine Translation\n",
        "1. Machine Translation is the task of converting a text from one language to another. In this context, we will focus on English to German (en-de) translation. The task involves processing a sequence of tokens in one language and producing a corresponding sequence of tokens in another language.\n",
        "2. We will use Encoder-Decoder Layer of Transformer\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "S0LeOzAKGm9S"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Let's Download the Data"
      ],
      "metadata": {
        "id": "SUiQQ3nCtDf2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import DataLoader\n",
        "from transformers import AutoTokenizer\n",
        "from datasets import load_dataset\n",
        "\n",
        "# Load the dataset and tokenizer\n",
        "dataset = load_dataset(\"iwslt2017\", \"iwslt2017-de-en\", split=\"train[:1%]\")  # Only use a 1% portion of the dataset\n",
        "tokenizer_src = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "tokenizer_tgt = AutoTokenizer.from_pretrained(\"bert-base-german-cased\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 361,
          "referenced_widgets": [
            "0079e93f38774ea59ea4e12f9026ca89",
            "45cedfc5ac0340ee954dbd5b8c5214e3",
            "587c8a1ef6e4423dacfd79445eeece3e",
            "ee1acbbf59b44c4f95281b4fcd9caa17",
            "0ffa7eb64d5f4c57b10585e132498af8",
            "77000c50e754461189db32d6f2ad0504",
            "d688073778854c42acf30d212b8cf051",
            "ab2ea25bb2bb4250b4797bcaa74cbd52",
            "7f8b7020948245ce96388074f0291c4b",
            "c7aa64302d1343c8bc4a039c5da3f89d",
            "68a9b65ed92743798d88bb7d8c3a8337",
            "55eacaeb314c41928c656375dbec7ecd",
            "121c357301284a279017f21ac1d20de2",
            "63a76ff7cb0d4f728645bfd2f6309c55",
            "1ea4ae4b10b745dfbeea153487644c27",
            "17aee5d00c25468c8369e091550ac030",
            "9717f61134df4bc1b03c2545d6360cb6",
            "99b02bb03e564523af650c57cdb526cf",
            "83fd6a9bcbcb4993a253af7a4525a1bd",
            "f5d58eb555b143e890a814607acee638",
            "b9b1db4ee56141cb9658fa95d56f518c",
            "6593d2518a6d49038852a28abe90075c",
            "5ddac0c7d61a4744a77fe779251d9992",
            "2fbaade58f664e26a5f65514311af29c",
            "97757d25dbee4a08ba909c067b43d675",
            "fc299d5d05c847489b720b1f662e695b",
            "13c3347b7bf34dda818903a5cbec0dab",
            "80f668c404b34d58b1f2a9595545d1b7",
            "735e20651b57481f8902e11b9050f041",
            "4bf99c47ffdf447ca592a748988aeb39",
            "4e36e31262e24608be8d72fcb9ee6d65",
            "343d0ae065304384a33e5c07479ae724",
            "2f3eca8ddec74681ac58af5a7e004160",
            "cfa511fba9a8412e83f3be96f02cbc95",
            "2a832f046e554c00acbc7559f6104550",
            "b4c146f0f0194dc99f8d28ea3ae01719",
            "ce0af431d90047ea9ba4255809220a5b",
            "c4cc841254364ac7aae6de12dbe858c8",
            "e4746c7989454b588dcdbe4aa347593d",
            "18b0d4c398c346f78edd84926699c86d",
            "7383f1b50747460db4fb2a841aaac9e7",
            "1979b206b4d84176ab7e2e41f4870deb",
            "14925890f80a43e6a125e9957e509228",
            "b63a5e406bfe4cec8abff8fd31938fb5",
            "2ec6b9ce358c45f883ec9b7e93d4bd21",
            "2b4132f308274e2fa64e70383632c348",
            "a7ebead26e2a4ea19ccc3c10a06413a4",
            "f24d28b7c2da47cf84f206f14335072f",
            "897783f78ead41408457716c8d426e19",
            "6cd672d1d49b4780881b6483e7f22763",
            "01946d4ce86b4a20aed3ed599962aa11",
            "e0652c6130864cdc955f76c856eab887",
            "0a778f305d214b77990ee3927edb6bbe",
            "3e9594efa6d04144829116ba877838ce",
            "37b6e138f0714db0bb8efc4a7b475217",
            "c8df6ae771c14d25a0938f025b8fe28d",
            "d0d8a3e2968b4836b06b00932d214c52",
            "f7f9c83502204976835843f7921cf382",
            "817c7b2032f14ea6bed78fb502ed22ee",
            "b3e3b6542744446987737c97f5294db0",
            "d855971788dc4d6eb501e84f146acbad",
            "cfa49ee604e54333a1e41d2c8800fd6e",
            "f69ac5e723f94865b0b60bb191acfc91",
            "4436339f386a4bc2a657131382b0deae",
            "b0bd4b6aaf48426ba62f6f42c9465cdc",
            "66dd3b97c7bd43cb85b62894f9c6451b",
            "f83863fcd03644088489bf1e37db18df",
            "c80756835d7c4d4ba7d317c231ff1458",
            "6d0be7b97a144a199f869723f601cb4e",
            "d0061ae7f41c473382bf8530a070d8f0",
            "f1e75741265d4fb7b3456ad59de108fd",
            "668df119ec6e4d129c7bd9feb5ffba9b",
            "08e2b3be0cc049a69ef1f23c56336761",
            "b6bf47ad411842fb94f63864d7854e40",
            "2ef31812ea784731b9e060ade9f5bd79",
            "1ef87883ceb8496486a41b77e28d5824",
            "128aa88c81b6466ba773d7d0de90b93b",
            "ffcd03a5d5e748feae7293e291daa399",
            "c4e214bc2e91443b8ac5cde5de97c704",
            "cbdc17b29ad1430ebdaf92e3adb61434",
            "e92947565c4d412fad22038072c3a7b6",
            "c073b9e046394f0998460e50939f2333",
            "45d1ed8ec10e4a7eaab365e5b273ddf9",
            "262b879df80b493787970f603f50388f",
            "5faccde4837c4fcab7b750e39a6a0c87",
            "8c7bb2f1888146cb85c059145d13aebf",
            "36825e58c18445e3a2094f165f7d32dc",
            "a1eeb4106ab7423a8f5eddd086f17a7c",
            "8e40600986ac49bba5ce7e23d35c341a",
            "72f6b4cbac20482db0ec36951dce4cd9",
            "095038d110b04b88bd0f0ec19f333dc4",
            "f569ed617bd94d92a909d8c83d63c7b2",
            "312071c63dc54e2284c14b618260bade",
            "8e8d2973aa6345c98e58e0f5cd73f69c",
            "3d1d1b34f905455ab86f836fc3f6aef3",
            "15011dd37b344df3ba8af0cce04dc348",
            "1b0259af9b0f492eb1f70ce56994e80a",
            "12399f28e4fe4bfc83307fd4481dacfa",
            "59f2fa9d24b944379951ed4920437ba6"
          ]
        },
        "id": "M3_12mzqKME1",
        "outputId": "26a4a8b4-a8ea-42a3-955a-b29509e2e261"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset iwslt2017/iwslt2017-de-en to /Users/premtimsina/.cache/huggingface/datasets/iwslt2017/iwslt2017-de-en/1.0.0/03ce9110373117c6f6687719f49f269486a8cd49dcad2527993a316cd4b6ad49...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/16.8M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0079e93f38774ea59ea4e12f9026ca89"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/206112 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "55eacaeb314c41928c656375dbec7ecd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split:   0%|          | 0/8079 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5ddac0c7d61a4744a77fe779251d9992"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating validation split:   0%|          | 0/888 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cfa511fba9a8412e83f3be96f02cbc95"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset iwslt2017 downloaded and prepared to /Users/premtimsina/.cache/huggingface/datasets/iwslt2017/iwslt2017-de-en/1.0.0/03ce9110373117c6f6687719f49f269486a8cd49dcad2527993a316cd4b6ad49. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2ec6b9ce358c45f883ec9b7e93d4bd21"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c8df6ae771c14d25a0938f025b8fe28d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/433 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f83863fcd03644088489bf1e37db18df"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/255k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ffcd03a5d5e748feae7293e291daa399"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/485k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8e40600986ac49bba5ce7e23d35c341a"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J1SVoXMDxQh2",
        "outputId": "ecedff1f-fdbc-457d-c41d-ddef22aa78b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'translation': {'de': 'Vielen Dank, Chris.',\n",
              "  'en': 'Thank you so much, Chris.'}}"
            ]
          },
          "metadata": {},
          "execution_count": 323
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Custom Dataset to Prepare Data\n",
        "1. Let's Prepare the data so that we have src_tokem and tgt_token of same length\n",
        "2. Here, we define the max length of token is 50"
      ],
      "metadata": {
        "id": "PQLQhdOixJjX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class TranslationDataset(Dataset):\n",
        "    def __init__(self, dataset, tokenizer_src, tokenizer_tgt, max_length=50):\n",
        "        self.dataset = dataset\n",
        "        self.tokenizer_src = tokenizer_src\n",
        "        self.tokenizer_tgt = tokenizer_tgt\n",
        "        self.max_length = max_length\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.dataset)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        src_text = self.dataset[idx]['translation']['en']\n",
        "        tgt_text = self.dataset[idx]['translation']['de']\n",
        "\n",
        "        src_tokens = self.tokenizer_src.encode_plus(\n",
        "            src_text, \n",
        "            max_length=self.max_length,\n",
        "            padding=\"max_length\",\n",
        "            truncation=True,\n",
        "            return_tensors=\"pt\",\n",
        "        )\n",
        "\n",
        "        tgt_tokens = self.tokenizer_tgt.encode_plus(\n",
        "            tgt_text,\n",
        "            max_length=self.max_length,\n",
        "            padding=\"max_length\",\n",
        "            truncation=True,\n",
        "            return_tensors=\"pt\",\n",
        "        )\n",
        "\n",
        "        return src_tokens[\"input_ids\"].squeeze(),tgt_tokens[\"input_ids\"].squeeze()\n"
      ],
      "metadata": {
        "id": "nFHDcmuR1bb5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = TranslationDataset(dataset, tokenizer_src, tokenizer_tgt)\n"
      ],
      "metadata": {
        "id": "sJUXR6Bmx05O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# View Sample data after creating torch.tensor\n",
        "train_data[2]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XZng6T8A1Sox",
        "outputId": "6dd91315-f550-4d7f-e085-1e905cd6631a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([  101,  1045,  2031,  2042, 10676,  2185,  2011,  2023,  3034,  1010,\n",
              "          1998,  1045,  2215,  2000,  4067,  2035,  1997,  2017,  2005,  1996,\n",
              "          2116,  3835,  7928,  2055,  2054,  1045,  2018,  2000,  2360,  1996,\n",
              "          2060,  2305,  1012,   102,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0]),\n",
              " tensor([    3,  1671,  4058,  4899, 15227,    88,   534, 13854, 26918,    42,\n",
              "          1169,  9334, 26897,  9830,  2122,   142,    30,  2709,  2055,  2636,\n",
              "             7, 18930,    81, 10183,  4468,  2085,  6738,  4253, 26914,     4,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0]))"
            ]
          },
          "metadata": {},
          "execution_count": 324
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Analysis: If you look at the above sample, `0` represents the padding token"
      ],
      "metadata": {
        "id": "y79RLQjqxwKs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create DataLoader:\n",
        "* The output after the dataloder should be of shape (seq_len, batch_size)\n",
        "* Thus, we have transposed both src_ids, and tgt_ids"
      ],
      "metadata": {
        "id": "m7wRvFs1yXiL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def collate_fn(batch):\n",
        "    src_ids ,tgt_ids = zip(*batch)\n",
        "    src_ids = torch.stack(src_ids).transpose(0, 1)\n",
        "    tgt_ids = torch.stack(tgt_ids).transpose(0, 1)\n",
        "    return src_ids, tgt_ids\n",
        "dataloader = DataLoader(train_data, batch_size=16, shuffle=True, collate_fn=collate_fn)\n"
      ],
      "metadata": {
        "id": "zlV3mT6o5A8g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "item=next(iter(dataloader))\n",
        "src_ids,tgt_ids=item\n",
        "print('src_ids ',src_ids.shape)\n",
        "print(' tgt_ids ',tgt_ids.shape)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mNWSlh-y66nP",
        "outputId": "d5f7dc93-0473-44da-98ba-01aa5d912b06"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "src_ids  torch.Size([50, 16])\n",
            " tgt_ids  torch.Size([50, 16])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "It looks perfect:\n",
        "1. the shape of source and target are [seq_length, batch_size]\n"
      ],
      "metadata": {
        "id": "QEn5D1vS86YY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torchtext\n",
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self, dim_embedding, dropout=0.1, max_seq_len=5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "        postional_encoding = torch.zeros(max_seq_len, dim_embedding)\n",
        "        position = torch.arange(0, max_seq_len, dtype=torch.float).unsqueeze(1)\n",
        "        denom_term = torch.exp(torch.arange(0, dim_embedding, 2).float() * (-math.log(10000.0) / dim_embedding))\n",
        "        postional_encoding[:, 0::2] = torch.sin(position * denom_term)\n",
        "        postional_encoding[:, 1::2] = torch.cos(position * denom_term)\n",
        "        postional_encoding = postional_encoding.unsqueeze(0).transpose(0, 1)\n",
        "        self.register_buffer('postional_encoding', postional_encoding)\n",
        "    def forward(self, x):\n",
        "        x = x + self.postional_encoding[:x.size(0), :]\n",
        "        return self.dropout(x)"
      ],
      "metadata": {
        "id": "R3Bm4meI2DqG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Transformer Model\n",
        "What we are doing?\n",
        "1. `forward`\n",
        "  * The source and target sequences are embedded and scaled by the square root of the embedding dimension.\n",
        "  * The positional encodings are added to the embeddings.\n",
        "  * The Transformer processes the source and target sequences, with masking\n",
        "    1. src_mask, trg_mask==> This is done to prevent future flow of information\n",
        "    2. src_padding_mask, trg_padding_mask ==> This is done to mask padded data. We are doing this so that model donot attent to padded tokens\n",
        "  * The output of the Transformer is passed through a fully connected layer to get the predicted target sequence..\n",
        "  * The model is predicting next token in german given all the tokens in en, and tokens untill the current step in german.\n",
        "  "
      ],
      "metadata": {
        "id": "q-VTmjS5ycak"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class TransformerModel(nn.Module):\n",
        "    def __init__(self,num_encoder_layers, num_decoder_layers, d_model, nhead, src_vocab_size=tokenizer_src.vocab_size, tgt_vocab_size=tokenizer_tgt.vocab_size, dim_feedforward=512, dropout=0.1):\n",
        "        super(TransformerModel, self).__init__()\n",
        "        self.src_embedding = nn.Embedding(input_dim, d_model)\n",
        "        self.trg_embedding = nn.Embedding(output_dim, d_model)\n",
        "        self.src_pos_encoder = PositionalEncoding(d_model, dropout)\n",
        "        self.trg_pos_encoder = PositionalEncoding(d_model, dropout)\n",
        "        self.transformer = nn.Transformer(d_model=d_model, nhead=nhead, num_encoder_layers=num_encoder_layers, num_decoder_layers=num_decoder_layers,dim_feedforward=dim_feedforward, dropout=dropout)\n",
        "        self.fc = nn.Linear(d_model, tgt_vocab_size)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.d_model = d_model\n",
        "\n",
        "    def forward(self, src, trg, src_mask=None, src_padding_mask=None,trg_mask=None, trg_padding_mask=None, memory_key_padding_mask=None):\n",
        "\n",
        "        src = self.src_embedding(src) * (self.d_model ** 0.5)\n",
        "        src = self.src_pos_encoder(src)\n",
        "        trg = self.trg_embedding(trg) * (self.d_model ** 0.5)\n",
        "        trg = self.trg_pos_encoder(trg)\n",
        "        output = self.transformer(src, trg,src_mask, trg_mask, None,\n",
        "                                src_padding_mask, trg_padding_mask, memory_key_padding_mask)\n",
        "        output = self.fc(self.dropout(output))\n",
        "        return output"
      ],
      "metadata": {
        "id": "F2_Yt_Nv1sSA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Let's Creating Masking\n",
        "1. `generate_square_subsequent_mask` will create following matrix for the tgt_msk. Here, we supposed the tgt sequence length is 3.\n",
        "\n",
        "```\n",
        "tensor([[ 0., -inf, -inf],\n",
        "        [ 0.,  0., -inf],\n",
        "        [ 0.,  0.,  0.]])\n",
        "```\n",
        "\n",
        "\n",
        "2. This means that each token would only be allowed to attend to the tokens that have already been generated during decoding.\n",
        "3. `src_mask` square matrix fill with False. This means, we are not masking any source sequence\n",
        "4. `src_padding_mask` and `tgt_padding_mask`: It looks at the src and tgt which is filled with padded token. And, mask all padded tokens\n",
        "5. We transpose `src_padding_mask` and `tgt_padding_mask` because Transformer requires the dimension to be [batch_size, seq_len]. This is opposite to the Transformer requirement for src and tgt to be [seq_length, batch_size]\n"
      ],
      "metadata": {
        "id": "Onvy1NPD1cEb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_square_subsequent_mask(sz):\n",
        "    mask = (torch.triu(torch.ones((sz, sz), device=device)) == 1).transpose(0, 1)\n",
        "    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
        "    return mask.to(device)\n",
        "\n",
        "\n",
        "def create_mask(src, tgt):\n",
        "    src_seq_len = src.shape[0]\n",
        "    tgt_seq_len = tgt.shape[0]\n",
        "\n",
        "    tgt_mask = generate_square_subsequent_mask(tgt_seq_len)\n",
        "    src_mask = torch.zeros((src_seq_len, src_seq_len),device=device).type(torch.bool)\n",
        "\n",
        "    src_padding_mask = (src == tokenizer_src.pad_token_id).transpose(0, 1)\n",
        "    tgt_padding_mask = (tgt == tokenizer_tgt.pad_token_id).transpose(0, 1)\n",
        "    return src_mask.to(device), tgt_mask.to(device), src_padding_mask.to(device), tgt_padding_mask.to(device)"
      ],
      "metadata": {
        "id": "lPEnR9_0ENgP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "generate_square_subsequent_mask(3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GqXHD6MW25BE",
        "outputId": "c1514ef4-ece7-4d32-f3e0-a02e3ac18b88"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0., -inf, -inf],\n",
              "        [0., 0., -inf],\n",
              "        [0., 0., 0.]], device='mps:0')"
            ]
          },
          "metadata": {},
          "execution_count": 327
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Let's Intiate Model and Conduct Training\n",
        "1. `nn.CrossEntropyLoss(ignore_index=tokenizer_tgt.pad_token_id)`\n",
        "  * We are asking loss function to ignore where it is padded token\n",
        "2. `tgt_out = tgt[1:, :]`\n",
        "\n",
        "  * we are removing the first token of the target sequence, since it corresponds to the special start-of-sentence token <sos>. By removing this token, we obtain a new tensor tgt_out that contains the remaining tokens of the target sequence, which will be used as input to the decoder during training. This is because during training, we want the model to learn to generate the target sequence given the input source sequence, without being provided with the start-of-sentence token\n",
        "\n",
        "3. ` loss=criterion(output.reshape(-1, output.shape[-1]), tgt_out.reshape(-1))\n",
        "`\n",
        "  * The output tensor has shape [tgt_seq_len - 1, batch_size, tgt_vocab_size], which means it has 3 dimensions. To calculate the loss, we need to reshape it to a 2D tensor of shape [(tgt_seq_len - 1) * batch_size, tgt_vocab_size].\n",
        "\n",
        "  * Similarly, tgt_out tensor has shape [tgt_seq_len - 1, batch_size], but to calculate the loss, we need to flatten it into a 1D tensor of shape [(tgt_seq_len - 1) * batch_size]."
      ],
      "metadata": {
        "id": "q-iOyj7a5kBG"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "m1XBYMmg5jna"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Instantiate the model\n",
        "input_dim = 50\n",
        "output_dim=50\n",
        "emb_size=512\n",
        "nhead = 8\n",
        "num_encoder_layers =num_decoder_layers= 3\n",
        "\n",
        "model = TransformerModel(num_encoder_layers, num_decoder_layers, emb_size, nhead, src_vocab_size=tokenizer_src.vocab_size, tgt_vocab_size=tokenizer_tgt.vocab_size, dim_feedforward=512).to(device)\n",
        "\n",
        "# Training loop\n",
        "num_epochs = 1\n",
        "criterion = nn.CrossEntropyLoss(ignore_index=tokenizer_tgt.pad_token_id)\n",
        "optimizer = optim.Adam(model.parameters())\n",
        "model.train()\n",
        "losses = 0\n",
        "for epoch in range(num_epochs):\n",
        "    for i, (src_ids, tgt_ids) in enumerate(dataloader):\n",
        "        src=src_ids.to(device)\n",
        "        tgt=tgt_ids.to(device)\n",
        "        tgt_input = tgt[:-1, :]\n",
        "        src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input)\n",
        "        # (self, src, trg, src_mask=None, src_padding_mask=None,trg_mask=None, trg_padding_mask=None, memory_key_padding_mask=None)\n",
        "        output=model(src, tgt_input, src_mask, src_padding_mask,tgt_mask, tgt_padding_mask, src_padding_mask)\n",
        "        optimizer.zero_grad()\n",
        "        tgt_out = tgt[1:, :]\n",
        "        loss=criterion(output.reshape(-1, output.shape[-1]), tgt_out.reshape(-1))\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        if i%100==0:\n",
        "          print('epoch ', epoch, 'batch ', i, ' loss ', loss)\n",
        "        losses=loss.item()\n",
        "    print( losses / float(len(list(dataloader))))\n",
        "\n",
        "\n",
        "\n",
        "        "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D9MBgnND3HqF",
        "outputId": "83641af6-87a2-4d2b-f32c-452dcdec4760"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/Users/premtimsina/opt/anaconda3/envs/transformer_learn/lib/python3.11/site-packages/torch/nn/functional.py:5007: UserWarning: Support for mismatched key_padding_mask and attn_mask is deprecated. Use same type for both instead.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch  0 batch  0  loss  tensor(10.5033, device='mps:0', grad_fn=<NllLossBackward0>)\n",
            "epoch  0 batch  100  loss  tensor(7.1293, device='mps:0', grad_fn=<NllLossBackward0>)\n",
            "0.05702095623164214\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Analysis:\n",
        "The above code is a simplified version of a machine translation model. In future chapters, we will explore more advanced models."
      ],
      "metadata": {
        "id": "RwnMVKAX_wWQ"
      }
    }
  ]
}